
import requests
import os
from bs4 import BeautifulSoup


def getProxy2File():
    os.chdir(r"./data/proxy")
    headers = {
        'User-Agent': 'Mozilla/5.0 (Linux; Android 6.0; Nexus 5 Build/MRA58N) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/56.0.2924.87 Mobile Safari/537.36'}
    fp = open("host.txt", "w")
    for i in range(1,2):
        url = 'http://www.xicidaili.com/nn/'+i.__str__()

        s = requests.get(url, headers=headers)

        soup = BeautifulSoup(s.text, 'lxml')
        ips = soup.select("#ip_list tr")
        for i in ips:
            try:
                ipp = i.select("td")
                ip = ipp[1].text
                host = ipp[2].text
                fp.write(ip)
                fp.write("\t")
                fp.write(host)
                fp.write("\n")
            except Exception as e:
                pass

    fp.close()

if __name__ == '__main__':
    getProxy2File()